package com.xiaotu.spider.project.pageprocessor;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.MathUtil;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.spider.APageProcessor;
import com.xiaotu.spider.downloader.SpiderHttpClientDownloader;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 WeiboFansPageProcessor
 * @日期 2017年5月22日
 * @作者 高海军
 * @功能 微博粉丝数据爬取
 */
@Component("WeiboFansPageProcessor")
@Scope("prototype")
public class WeiboFansPageProcessor extends APageProcessor
{
	private static final String ESCAPE_TOPICID = "#topicId#";
	private static final String ESCAPE_PAGENO = "#pageNo#";
	private static final String ESCAPE_LASTPAGENO = "#lastPageNo#";
	
	private static final String FANS_LIST_URL = "http://weibo.com/p/"
			+ ESCAPE_TOPICID
			+ "/followlist?pids=Pl_Core_F4RightUserList__39&page="
			+ ESCAPE_PAGENO + "&ajaxpagelet=1&ajaxpagelet_v6=1&__ref=/p/"
			+ ESCAPE_TOPICID + "/followlist?page=" + ESCAPE_LASTPAGENO
			+ "#Pl_Core_F4RightUserList__39&_t=FM_";
	
	private static final String TOPIC_URL_STAR = "http://weibo.com/p/";
	private static final String TOPIC_URL_END = "/followlist";
	
	private static final String SCRIPT_PARENT_START = "<script>parent.FM.view(";
	private static final String SCRIPT_FM_START = "<script>FM.view(";
	private static final String SCRIPT_END = ")</script>";
	
	private static final String LABELS_LABEL = "标签信息";
	private static final String LABELS_EDU = "教育信息";
	private static final String LABELS_JOB = "工作信息";
	private static final String LABELS_BASIC = "基本信息";
	
	private static final List<String> LABELS_ALL = Arrays.asList("所在地", "性别",
			"生日", "简介", LABELS_LABEL, LABELS_EDU, LABELS_JOB);
	
	private String fileName;
	private int maxPageNo = 10;
	
	@Override
	public void initRunParams(Map<String, Object> map)
	{
		if (map.get("fileName") == null)
			throw new SpiderException(
					SpiderExceptionCode.PageProcessorParameterErrorException);
		fileName = FileUtil
				.getCacheFilePath(map.get("fileName") + StringUtils.EMPTY);
		if (map.get("pageNo") != null)
			maxPageNo = Integer.parseInt(map.get("pageNo") + StringUtils.EMPTY);
	}
	
	@Override
	protected Request getStartRequest(JSONObject target)
	{
		target.put(ESCAPE_TOPICID, this.getTopicId(target.getString("url")));
		target.put(ESCAPE_PAGENO, 1);
		target.putAll(this.getOtherTarget());
		return new Request(this.getUrl(target));
	}
	
	/**
	 * 获取粉丝列表的url地址
	 * @param target 话题地址
	 * @return url
	 */
	private String getUrl(JSONObject target)
	{
		int pageNo = target.getInt(ESCAPE_PAGENO);
		String pageNoSuf = StringUtils.EMPTY;
		if (pageNo > 2)
			pageNoSuf = "?page=" + (pageNo - 1);
		target.put(SpiderHttpClientDownloader.HEADER_REFERER,
				"http://weibo.com/p/" + target.getString(ESCAPE_TOPICID)
						+ "/followlist" + pageNoSuf);
		String url = FANS_LIST_URL
				.replaceAll(ESCAPE_TOPICID, target.getString(ESCAPE_TOPICID))
				.replaceAll(ESCAPE_PAGENO, pageNo + StringUtils.EMPTY)
				.replaceAll(ESCAPE_LASTPAGENO, (pageNo - 1) + StringUtils.EMPTY)
				+ System.currentTimeMillis() + StringUtils.EMPTY
				+ StringUtils.leftPad(
						MathUtil.getRand(1, 99) + StringUtils.EMPTY, 2, "0");
		String ref = "__ref=";
		int start = url.indexOf(ref) + ref.length();
		int end = start + url.substring(start).indexOf("&");
		url = url.substring(0, start)
				+ KeywordEncode.encode(url.substring(start, end))
				+ url.substring(end);
		return url;
	}
	
	/**
	 * 从url中解析出话题id
	 * @param url 话题url
	 * @return 话题id
	 */
	private String getTopicId(String url)
	{
		if (!RegexUtils.regexMatch(TOPIC_URL_STAR + "([^/]*)" + TOPIC_URL_END
				+ RegexUtils.REGEX_SUFFIX, url))
			throw new SpiderException(SpiderExceptionCode.DataInvalideException,
					url);
		String topicId = url.replace(TOPIC_URL_STAR, StringUtils.EMPTY)
				.replace(TOPIC_URL_END, StringUtils.EMPTY);
		if (StringUtils.isEmpty(topicId))
			throw new SpiderException(SpiderExceptionCode.DataInvalideException,
					url);
		return topicId;
	}
	
	@Override
	public void process(Page page)
	{
		if (page.getUrl().toString().contains(TOPIC_URL_END))
			this.fansListProcess(page);
		else
			this.fansProcess(page);
	}
	
	/**
	 * 基本信息解析
	 * @param html 页面元素
	 * @param map 缓存
	 */
	private void parseBaseinfo(Html html, Map<String, String> map)
	{
		List<String> liList = html.xpath("//li[@class='li_1 clearfix']").all();
		if (liList == null || liList.isEmpty())
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		for (String liStr : liList)
		{
			Html liObj = new Html(liStr);
			String label = liObj
					.xpath("//span[@class='pt_title S_txt2']/text()").get();
			label = label.replace(SepratorUtil.SEP_COLON_CN, StringUtils.EMPTY);
			if (LABELS_ALL.contains(label))
				map.put(label,
						liObj.xpath("//span[@class='pt_detail']/text()").get());
		}
	}
	
	/**
	 * 标签信息解析
	 * @param html 页面元素
	 * @param map 缓存
	 */
	private void parseLabel(Html html, Map<String, String> map)
	{
		List<String> aList = html.xpath("//a/text()").all();
		if (aList == null || aList.isEmpty())
			return;
		StringBuffer buffer = new StringBuffer();
		for (String a : aList)
			buffer.append(SepratorUtil.SEP_COMMA_EN + a.trim());
		map.put(LABELS_LABEL,
				buffer.substring(SepratorUtil.SEP_COMMA_EN.length()));
	}
	
	/**
	 * 其他信息解析
	 * @param html 页面元素
	 * @param map 缓存
	 * @param label 要解析的信息名称
	 */
	private void parseHtml(Html html, Map<String, String> map, String label)
	{
		map.put(label, HtmlUtil
				.delHTMLTag(html.xpath("//span[@class='pt_detail']").get()));
	}
	
	/**
	 * 即系粉丝的详细信息
	 * @param html 页面元素
	 */
	private void parseInfo(String userId, Html html)
	{
		List<String> divList = html.xpath("//div[@class='WB_cardwrap S_bg2']")
				.all();
		if (divList == null || divList.isEmpty())
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		Map<String, String> map = new HashMap<String, String>();
		boolean ok = false;
		for (String divStr : divList)
		{
			Html divObj = new Html(divStr);
			String label = divObj.xpath("//div[@class='obj_name']/h2/text()")
					.get();
			switch (label)
			{
				case LABELS_BASIC:
					this.parseBaseinfo(divObj, map);
					ok = true;
					break;
				case LABELS_LABEL:
					this.parseLabel(divObj, map);
					break;
				case LABELS_EDU:
				case LABELS_JOB:
					this.parseHtml(divObj, map, label);
					break;
			}
		}
		
		if (!ok)
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		
		StringBuffer buffer = new StringBuffer(userId);
		for (String key : LABELS_ALL)
			buffer.append(SepratorUtil.SEP_PIPE + (map.containsKey(key)
					? map.get(key) : StringUtils.EMPTY));
		FileUtil.writeFile(fileName,
				buffer.toString() + SepratorUtil.LINE_SEPRATOR);
	}
	
	/**
	 * 粉丝详情处理
	 * @param page 页面对象
	 */
	private void fansProcess(Page page)
	{
		int idx = 1;
		String response = null;
		while (true)
		{
			int start = StringUtils.lastOrdinalIndexOf(page.getRawText(),
					SCRIPT_FM_START, idx);
			if (start < 0)
				break;
			int end = StringUtils.lastOrdinalIndexOf(page.getRawText(),
					SCRIPT_END, idx);
			
			String temp = StringUtils.substring(page.getRawText(),
					start + SCRIPT_FM_START.length(), end);
			if (temp.contains("基本信息"))
			{
				response = temp;
				break;
			}
			idx++;
		}
		if (StringUtils.isEmpty(response))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		JSONObject jsonObj = JSONObject.fromObject(response);
		if (jsonObj == null || jsonObj.isEmpty() || !jsonObj.has("html"))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		Html divObj = new Html(jsonObj.getString("html"));
		this.parseInfo(page.getRequest().getExtra("userId") + StringUtils.EMPTY,
				divObj);
	}
	
	/**
	 * 粉丝列表处理：添加粉丝详情页面请求
	 * @param page 页面对象
	 */
	@SuppressWarnings("unchecked")
	private void fansListProcess(Page page)
	{
		String response = page.getRawText();
		if (!response.startsWith(SCRIPT_PARENT_START))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		
		response = response.replace(SCRIPT_PARENT_START, StringUtils.EMPTY)
				.replace(SCRIPT_END, StringUtils.EMPTY);
		JSONObject jsonObj = JSONObject.fromObject(response);
		if (jsonObj == null || jsonObj.isEmpty() || !jsonObj.has("html"))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		Html html = new Html(jsonObj.getString("html"));
		List<String> liList = html.xpath("//li[@class='follow_item S_line2']")
				.all();
		for (String divStr : liList)
		{
			String userId = new Html(divStr)
					.xpath("//a[@class='S_txt1']/strong/@usercard").get();
			userId = RegexUtils.regexSearch("^id=" + RegexUtils.REGEX_INTEGER,
					userId);
			if (StringUtils.isEmpty(userId))
				continue;
			userId = userId.replace("id=", StringUtils.EMPTY);
			page.getRequest().putExtra("userId", userId);
			PageUtils.addTargetRequest(page,
					"http://weibo.com/p/100505" + userId
							+ "/info?mod=pedit_more",
					page.getRequest().getExtras(), true);// 粉丝详情页面
		}
		
		Map<String, Object> map = page.getRequest().getExtras();
		if (maxPageNo > (int) map.get(ESCAPE_PAGENO))
		{
			map.put(ESCAPE_PAGENO, (int) map.get(ESCAPE_PAGENO) + 1);
			JSONObject target = JSONObject.fromObject(map);
			PageUtils.addTargetRequest(page, this.getUrl(target),
					new HashMap<String, Object>(target));// 粉丝列表下一页
		}
		
	}
	
	@Override
	protected String getSpiderType()
	{
		return "WeiboFans";
	}
	
	@Override
	public String getPageDataType(Page page)
	{
		return "WeiboFans";
	}
}
